import requests
import parsel
import tomd
import os
import pathvalidate  # 引入 pathvalidate 模块
import re

def spider_one_csdn(title_url, final_road):  
    head = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 Edg/84.0.522.52",
        "Referer": "https://blog.csdn.net/tansty_zh"
    }
    html = requests.get(url=title_url, headers=head).text
    page = parsel.Selector(html)
    title = page.css(".title-article::text").get()
    print(title)
    content = page.css("article").get()
    content = re.sub("<a.*?a>", "", content)
    content = re.sub("<br>", "", content)
    text = tomd.Tomd(content).markdown
    
    # 使用 pathvalidate 模块确保文件名有效
    cleaned_title = pathvalidate.sanitize_filename(title, platform='auto')
    filename = os.path.join(final_road, cleaned_title + ".md")
    # 将 Markdown 内容保存到文件中
    with open(filename, mode="w", encoding="utf-8") as f:
        f.write("#" + title)
        f.write(text)

def get_article_link(user, final_road):  
    page = 1
    head = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 Edg/84.0.522.52",
        "Referer": "https://blog.csdn.net/sunshineGGB"
    }
    while True:
        link = "https://blog.csdn.net/{}/article/list/{}".format(user, page)
        print("现在爬取第", page, "页")
        html = requests.get(url=link, headers=head).text
        cel = parsel.Selector(html)
        name_link = cel.css(".article-list h4 a::attr(href) ").getall()
        if not name_link:
            break
        for name in name_link:
            spider_one_csdn(name, final_road)
        page += 1

def main():
    print("本项目由tansty开发，GoodTime优化完善")
    name = input("请输入博主的名称：")
    file_name = "./passage"
    final_road = os.path.join(os.getcwd(), file_name)
    try:
        os.mkdir(final_road)
        print('创建成功！')
    except:
        pass
    get_article_link(name, final_road)

# https://blog.csdn.net/sunshineGGB/article/list/1
if __name__ == '__main__':
    main()
